from lxml import etree
import time
import requests
import csv
import random


def PageGenerator(n):
    return "https://beijing.anjuke.com/sale/p{}-rd1/#filtersort".format(n)


def SortOut(element):

    ele_house_title = element.xpath(".//div[@class='house-title']/a")[0]
    theme = ele_house_title.text.strip()
    link = ele_house_title.get('href')

    ele_details_item = element.xpath(".//div[@class='details-item']")[0]
    details = [x.text for x in ele_details_item.xpath(".//span")]
    if len(details) > 3:
        room, area, floor, age = details
    else:
        room, area, floor = details
        age = "Unknown"

    ele_comm_address = element.xpath(".//span[@class='comm-address']")[0]
    rag = ele_comm_address.text.strip().split()
    if len(rag) < 3:
        community, addr = rag
    else:
        community, *addr = rag

    price = element.xpath(".//span[@class='price-det']/strong")[0].text

    return{
        "theme": theme,
        "room": room,
        "area": area,
        "floor": floor,
        "age": age,
        "community": community,
        "addr": addr,
        "price": price,
        "link": link,
    }


if __name__ == "__main__":

    with open('beijing.csv', 'w', newline='') as csvfile:
        spamwriter = csv.writer(csvfile,
                                delimiter='|',
                                quotechar='^',
                                quoting=csv.QUOTE_MINIMAL)
        spamwriter.writerow(
            ("标题", "房间", "面积", "楼层", "年龄", "小区", "地址", "价格", "链接"))

        headers = {
            'user-agent':
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:80.0)' +
            ' Gecko/20100101 Firefox/80.0'
        }

        for i in range(20):
            page = PageGenerator(i+1)
            print("Page{} Starting".format(page))
            response = requests.get(page, headers=headers)
            html = response.content.decode("UTF-8")

            html = etree.HTML(html)
            html = html.xpath("//li[@class='list-item']")

            resultset = map(SortOut, html)
            resultset = list(resultset)

            for r in resultset:

                ln = (r["theme"], r["room"], r["area"], r["floor"], r["age"],
                      r["community"], r["addr"], r["price"], r["link"])
                spamwriter.writerow(ln)
                print("Written:{}".format(r["theme"]))

            time.sleep(random.randint(20, 30))
